﻿/*======================================================================
== Copyright : BlueCurve (c)
== Licence   : Gnu/GPL v2.x
== Author    : Teddy Albina
== Email     : bluecurveteam@gmail.com
== Web site  : http://www.codeplex.com/BlueCurve
========================================================================*/
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Encode.Detector;
using HtmlAgilityPack;

namespace BlueCurve.Common
{
    /// <summary>
    /// Fournit un ensemble de fonctions pour BlueCurve
    /// </summary>
    public class CommonFunctions
    {
        #region 'Configuration'

        /// <summary>
        /// Objet pour la détection d'encodage
        /// </summary>
        private EncodeDetector detector = new EncodeDetector();
        /// <summary>
        /// Regexp options
        /// </summary>
        private RegexOptions options = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase;
        
        #endregion


        #region 'URLS'

        /// <summary>
        /// Normalise un lien hypertexte
        /// </summary>
        /// <param name="url">Url à normaliser</param>
        /// <returns>string</returns>
        public string NormalizeLink(string url)
        {
            try
            {
                if (string.IsNullOrEmpty(url))
                    return null;

                Regex stripfragment = new Regex(@"[#](.*)$", RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
                Regex stringendslash = new Regex(@"/$", RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);

                string res = stripfragment.Replace(url, "");
                res = stringendslash.Replace(res, "");
                
                if (res.StartsWith("http://www.", StringComparison.InvariantCultureIgnoreCase))
                    res = Regex.Replace(res, @"http://www.", @"http://www.", RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
                else if (res.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
                    res = Regex.Replace(res, @"http://", @"http://", RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase);
                else
                    res = res.Insert(0, "http://");

                return res;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// Normalise un lien hypertexte
        /// </summary>
        /// <param name="url">Url de base</param>
        /// <param name="_url">Url à normaliser</param>
        /// <returns>string</returns>
        public virtual string NormalizeLink(string url, string _url)
        {
            try
            {
                return NormalizeLink(new Uri(new Uri(url), _url).ToString());
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Extrait le domaine à partir d'une url
        /// </summary>
        /// <param name="url">Url à partir de laquelle extraite le nom de domaine</param>
        /// <returns>string</returns>
        public string GetDomainFromLink(string url)
        {
            try
            {
                Uri uri = new Uri(url);
                Regex b = new Regex(@"^www\.", RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
                return b.Replace(uri.Host, "").ToLower();
            }
            catch
            {
                return null;
            }
        }

        #endregion
        
        
        #region 'NGRAMS'

        /// <summary>
        /// Détermine la langue de rédaction d'un document
        /// </summary>
        /// <param name="contentToTest">Texte à évaluer</param>
        /// <returns>string</returns>
        public string FindDocumentLanguage(ref string contentToTest)
        {
            try
            {
                SortedList<float, string> res = new SortedList<float, string>();
                foreach (KeyValuePair<string, List<string>> t in Common.Ngrams.NgramsDictionnaries.m_dictionnaries)
                {
                    float temp = Common.Ngrams.NGram.GetTrigramSimilarity(t.Value, contentToTest);
                    if (!res.ContainsKey(temp))
                        res.Add(temp, t.Key);
                }

                return res.Values[res.Count - 1];
            }
            catch
            {
                return "not found";
            }
        }

        #endregion

        
        #region 'Thumbnail'

        /// <summary>
        /// Extrait la miniature d'un fichier
        /// </summary>
        /// <param name="path">Chemin d'accès du fichier</param>
        /// <param name="height">Hauteur de la miniature</param>
        /// <param name="width">Largeur de la miniature</param>
        /// <returns>Bitmap</returns>
        public Bitmap ExtractThumbnail(string path, int height, int width)
        {
            try
            {
                FileThumbnail thumb = new FileThumbnail(path, width, height);
                return thumb.thumbBitmap;
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Extrait la miniature d'un fichier
        /// et l'écrit sur le disque
        /// </summary>
        /// <param name="path">Chemin d'accès du fichier</param>
        /// <param name="height">Hauteur de la miniature</param>
        /// <param name="width">Largeur de la miniature</param>
        /// <param name="savepath">Chemin de sauvegarde de la miniature</param>
        /// <returns>bool</returns>
        public virtual bool ExtractThumbnail(string path, int height, int width, string savepath)
        {
            try
            {
                FileThumbnail thumb = new FileThumbnail(path, width, height);
                thumb.thumbBitmap.Save(string.Concat(savepath, ".png"), System.Drawing.Imaging.ImageFormat.Png);

                return true;
            }
            catch
            {
                return false;
            }
        }

        #endregion
        

        #region 'Text'

        /// <summary>
        /// Filtre les accents d'un texte
        /// </summary>
        /// <param name="text">Texte à filtrer</param>
        /// <returns>string</returns>
        public string AccentFilter(string text)
        {
            try
            {
                StringBuilder output = new StringBuilder();
                for (int i = 0; i < text.Length; i++)
                {
                    long val = text[i];
                    switch (text[i])
                    {
                        case '\u00C0':  // Ã€
                        case '\u00C1':  // Ã?
                        case '\u00C2':  // Ã‚
                        case '\u00C3':  // Ãƒ
                        case '\u00C4':  // Ã„
                        case '\u00C5':  // Ã…
                            output.Append("A");
                            break;

                        case '\u00C6':  // Ã†
                            output.Append("AE");
                            break;

                        case '\u00C7':  // Ã‡
                            output.Append("C");
                            break;

                        case '\u00C8':  // Ãˆ
                        case '\u00C9':  // Ã‰
                        case '\u00CA':  // ÃŠ
                        case '\u00CB':  // Ã‹
                            output.Append("E");
                            break;

                        case '\u00CC':  // ÃŒ
                        case '\u00CD':  // Ã?
                        case '\u00CE':  // ÃŽ
                        case '\u00CF':  // Ã?
                            output.Append("I");
                            break;

                        case '\u00D0':  // Ã?
                            output.Append("D");
                            break;

                        case '\u00D1':  // Ã‘
                            output.Append("N");
                            break;

                        case '\u00D2':  // Ã’
                        case '\u00D3':  // Ã“
                        case '\u00D4':  // Ã”
                        case '\u00D5':  // Ã•
                        case '\u00D6':  // Ã–
                        case '\u00D8':  // Ã˜
                            output.Append("O");
                            break;

                        case '\u0152':  // Å’
                            output.Append("OE");
                            break;

                        case '\u00DE':  // Ãž
                            output.Append("TH");
                            break;

                        case '\u00D9':  // Ã™
                        case '\u00DA':  // Ãš
                        case '\u00DB':  // Ã›
                        case '\u00DC':  // Ãœ
                            output.Append("U");
                            break;

                        case '\u00DD':  // Ã?
                        case '\u0178':  // Å¸
                            output.Append("Y");
                            break;

                        case '\u00E0':  // Ã 
                        case '\u00E1':  // Ã¡
                        case '\u00E2':  // Ã¢
                        case '\u00E3':  // Ã£
                        case '\u00E4':  // Ã¤
                        case '\u00E5':  // Ã¥
                            output.Append("a");
                            break;

                        case '\u00E6':  // Ã¦
                            output.Append("ae");
                            break;

                        case '\u00E7':  // Ã§
                            output.Append("c");
                            break;

                        case '\u00E8':  // Ã¨
                        case '\u00E9':  // Ã©
                        case '\u00EA':  // Ãª
                        case '\u00EB':  // Ã«
                            output.Append("e");
                            break;

                        case '\u00EC':  // Ã¬
                        case '\u00ED':  // Ã­
                        case '\u00EE':  // Ã®
                        case '\u00EF':  // Ã¯
                            output.Append("i");
                            break;

                        case '\u00F0':  // Ã°
                            output.Append("d");
                            break;

                        case '\u00F1':  // Ã±
                            output.Append("n");
                            break;

                        case '\u00F2':  // Ã²
                        case '\u00F3':  // Ã³
                        case '\u00F4':  // Ã´
                        case '\u00F5':  // Ãµ
                        case '\u00F6':  // Ã¶
                        case '\u00F8':  // Ã¸
                            output.Append("o");
                            break;

                        case '\u0153':  // Å“
                            output.Append("oe");
                            break;

                        case '\u00DF':  // ÃŸ
                            output.Append("ss");
                            break;

                        case '\u00FE':  // Ã¾
                            output.Append("th");
                            break;

                        case '\u00F9':  // Ã¹
                        case '\u00FA':  // Ãº
                        case '\u00FB':  // Ã»
                        case '\u00FC':  // Ã¼
                            output.Append("u");
                            break;

                        case '\u00FD':  // Ã½
                        case '\u00FF':  // Ã¿
                            output.Append("y");
                            break;
                        default:
                            output.Append(text[i]);
                            break;
                    }
                }

                return output.ToString();
            }
            catch
            {
                return null;
            }
        }

        #endregion
        
        
        #region 'Functions'

        /// <summary>
        /// Obtiens le dernier élément d'un chemin de fichier
        /// pour <![CDATA[C:\Users\axelle\Desktop\Karima\jap.wb]]>
        /// la fonction retournera jap.wb
        /// </summary>
        /// <param name="path">Chemin de fichier à évaluer</param>
        /// <returns>string</returns>
        public string GetLastPathElementFromPath(string path)
        {
            try
            {
                Regex a = new Regex(@"\\?([^\\]+)$", RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
                return a.Match(path, 0).Value.Split('\\')[1];
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// Retourne le protocol utilisé par un URI
        /// ex > <![CDATA[http://www.free.fr]]> retournera http
        /// </summary>
        /// <param name="uri">Uri</param>
        /// <returns>string</returns>
        public string GetScheme(string uri)
        {
            return uri.Split(new string[] { "://" }, StringSplitOptions.RemoveEmptyEntries)[0].ToLower();
        }
               
        /// <summary>
        /// Lit un fichier en utf8
        /// </summary>
        private string ReadFile(string ntfs)
        {
            try
            {
                string data = null;
                using (StreamReader file = new StreamReader(File.Open(ntfs, FileMode.Open, FileAccess.Read, FileShare.None), Encoding.UTF8, this.DetectTagBom(ntfs)))
                {
                    data = file.ReadToEnd();
                    file.Close();
                }
                return data;
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Teste si un répertoire peut être parsé par le scheduler
        /// </summary>
        /// <param name="dir">Chemin de fichier à évaluer</param>
        /// <returns>bool</returns>
        public bool CheckDirectoryAttributes(string dir)
        {
            try
            {
                if (!Directory.Exists(dir))
                    return false;
                if ((File.GetAttributes(dir) & FileAttributes.Directory) != FileAttributes.Directory)
                    return false;
                if ((File.GetAttributes(dir) & FileAttributes.Archive) == FileAttributes.Archive
                    || (File.GetAttributes(dir) & FileAttributes.Encrypted) == FileAttributes.Encrypted
                    || (File.GetAttributes(dir) & FileAttributes.Hidden) == FileAttributes.Hidden
                    || (File.GetAttributes(dir) & FileAttributes.Offline) == FileAttributes.Offline
                    || (File.GetAttributes(dir) & FileAttributes.System) == FileAttributes.System
                    || (File.GetAttributes(dir) & FileAttributes.Temporary) == FileAttributes.Temporary)
                    return false;

                return true;
            }
            catch
            {
                return false;
            }
        }
        
        /// <summary>
        /// Detecte l'encodage d'un document en utilisant Encode.Detector
        /// </summary>
        /// <param name="ntfs">Chemin d'accès du fichier à évaluer</param>
        /// <returns>Encoding</returns>
        public Encoding DetectEncoding(string ntfs)
        {
            try
            {
                StreamReader read = detector.OpenTextStream(File.Open(ntfs, FileMode.Open, FileAccess.Read, FileShare.Read));
                Encoding c = detector.GetMostEfficientEncoding(read.ReadToEnd());
                read.Close();
                read.Dispose();
                return c;
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Supprime les espaces superflux dans un texte
        /// </summary>
        /// <param name="text">Texte à traiter</param>
        /// <returns>string </returns>
        public string Stripspaces(string text)
        {
            // clean spaces
            Regex clean = new Regex(@"^\s+");
                  text = clean.Replace(text, " ");
            clean = new Regex(@"\s+$");
                  text = clean.Replace(text, " ");
            clean = new Regex(@"\s+");
            return clean.Replace(text, " ");
        }
        
        /// <summary>
        /// Transcode document
        /// Convertie un document en utf-8 et remplace le fichier d'origine
        /// <see cref="http://blog.blueshop.com.tw/uni2tw/archive/2006/10/04/41342.aspx"/>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="ntfs">Chemin d'accès au document à convertir</param>
        /// </summary>
        public void ConvertEncodingReplace(int currentencoding, string ntfs)
        {
            // Conversion
            Stream fs = File.Open(ntfs, FileMode.Open, FileAccess.Read);
            byte[] raw = new byte[fs.Length];
            fs.Read(raw, 0, (int)fs.Length);
            byte[] utf8 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.UTF8, raw);
            fs.Close();
            fs.Dispose();

            // Replace file
            Stream str = File.Create(ntfs, utf8.Length, FileOptions.None);
            str.Write(utf8, 0, utf8.Length);
            str.Close();
            str.Dispose();
        }
        
        /// <summary>
        /// Convertie un document en utf-8
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="ntfs">Chemin d'accès au document à convertir</param>
        /// <returns>string</returns>
        public string ConvertEncodingToUtf8(int currentencoding, string ntfs)
        {
            // Conversion
            Stream fs = File.Open(ntfs, FileMode.Open, FileAccess.Read);
            byte[] raw = new byte[(int)fs.Length];
            fs.Read(raw, 0, (int)fs.Length);
            byte[] utf8 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.UTF8, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UTF8Encoding encutf8 = new UTF8Encoding();
            return encutf8.GetString(utf8);
        }
        
        /// <summary>
        /// Convertie un document en unicode utf16le
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="ntfs">Chemin d'accès au document à convertir</param>
        /// <returns>string</returns>
        public string ConvertEncodingToUtf16le(int currentencoding, string ntfs)
        {
            // Conversion
            Stream fs = File.Open(ntfs, FileMode.Open, FileAccess.Read);
            byte[] raw = new byte[(int)fs.Length];
            fs.Read(raw, 0, (int)fs.Length);
            byte[] utf16 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.Unicode, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UnicodeEncoding encutf16 = new UnicodeEncoding(false, false);
            return encutf16.GetString(utf16);
        }
        
        /// <summary>
        /// Convertie un document en unicode utf16be
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="ntfs">Chemin d'accès au document à convertir</param>
        /// <returns>string</returns>
        public string ConvertEncodingToUtf16be(int currentencoding, string ntfs)
        {
            // Conversion
            Stream fs = File.Open(ntfs, FileMode.Open, FileAccess.Read);
            byte[] raw = new byte[(int)fs.Length];
            fs.Read(raw, 0, (int)fs.Length);
            byte[] utf16 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.Unicode, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UnicodeEncoding encutf16 = new UnicodeEncoding(true, false);
            return encutf16.GetString(utf16);
        }
        
        /// <summary>
        /// Convertie un document en utf-8
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="content">Contenue du document</param>
        /// <returns>string</returns>
        public string ConvertEncodingFromMemoryToUtf8(int currentencoding, ref string content)
        { 
            // Conversion
            MemoryStream fs = new MemoryStream(Encoding.GetEncoding(currentencoding).GetBytes(content));
            byte[] raw = new byte[content.Length];
            fs.Read(raw, 0, content.Length);
            byte[] utf8 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.UTF8, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UTF8Encoding encutf8 = new UTF8Encoding();
            return encutf8.GetString(utf8);
        }
        
        /// <summary>
        /// Convertie un document en utf16le
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="content">Contenue du document</param>
        /// <returns>string</returns>
        public string ConvertEncodingFromMemoryToUtf16le(int currentencoding, ref string content)
        {
            // Conversion
            MemoryStream fs = new MemoryStream(Encoding.GetEncoding(currentencoding).GetBytes(content));
            byte[] raw = new byte[content.Length];
            fs.Read(raw, 0, content.Length);
            byte[] utf16 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.Unicode, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UnicodeEncoding encutf16 = new UnicodeEncoding(false, false);
            return encutf16.GetString(utf16);
        }
        
        /// <summary>
        /// Convertie un document en utf16be
        /// </summary>
        /// <param name="currentencoding">Codepage de l'encodage actuel du document</param>
        /// <param name="content">Contenue du document</param>
        /// <returns>string</returns>
        public string ConvertEncodingFromMemoryToUtf16be(int currentencoding, ref string content)
        {
            // Conversion
            MemoryStream fs = new MemoryStream(Encoding.GetEncoding(currentencoding).GetBytes(content));
            byte[] raw = new byte[content.Length];
            fs.Read(raw, 0, content.Length);
            byte[] utf16 = Encoding.Convert(Encoding.GetEncoding(currentencoding), Encoding.Unicode, raw);
            fs.Close();
            fs.Dispose();

            // Read
            UnicodeEncoding encutf16 = new UnicodeEncoding(true, false);
            return encutf16.GetString(utf16);
        }
        
        /// <summary>
        /// Retourne la longueur en byte d'un fichier
        /// </summary>
        /// <param name="location">Chemin d'accès du fichier à évaluer</param>
        /// <returns>Taille du fichier</returns>
        public float GetFileLength(string location)
        {
            try
            {
                FileInfo info = new FileInfo(location);
                return Convert.ToSingle(info.Length/1024);
            }
            catch
            {
                return 0;
            }
        }
        
        /// <summary>
        /// Detecte l'encodage d'un document
        /// en ce basant sur l'analyse des premiers bytes
        /// <param name="ntfs">Chemin d'accès du fichier à évaluer</param>
        /// </summary>
        /// <returns>Encoding</returns>
        public Encoding GetFileEncoding(string ntfs)
        {
            Encoding enc = null;
            try
            {
                FileStream file = new FileStream(ntfs, FileMode.Open, FileAccess.Read, FileShare.Read);
                if (file.CanSeek)
                {
                    byte[] bom = new byte[4]; // Get the byte-order mark, if there is one
                    file.Read(bom, 0, 4);
                    if (bom[0] == 0xef && bom[1] == 0xbb && bom[2] == 0xbf) // utf-8
                    {
                        enc = Encoding.UTF8;
                    }
                    else if ((bom[0] == 0xff && bom[1] == 0xfe) || // ucs-2le, ucs-4le, and ucs-16le
                        (bom[0] == 0 && bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff)) // ucs-4
                    {
                        enc = Encoding.Unicode;
                    }
                    else if (bom[0] == 0xfe && bom[1] == 0xff) // utf-16 and ucs-2
                    {
                        enc = Encoding.BigEndianUnicode;
                    }
                    else // ANSI, Default
                    {
                        enc = null;
                    }
                    // Now reposition the file cursor back to the start of the file
                    file.Seek(0, SeekOrigin.Begin);
                }
                else
                {
                    // The file cannot be randomly accessed, so you need to decide what to set the default to
                    // based on the data provided. If you're expecting data from a lot of older applications,
                    // default your encoding to Encoding.ASCII. If you're expecting data from a lot of newer
                    // applications, default your encoding to Encoding.Unicode. Also, since binary files are
                    // single byte-based, so you will want to use Encoding.ASCII, even though you'll probably
                    // never need to use the encoding then since the Encoding classes are really meant to get
                    // strings from the byte array that is the file.

                    enc = Encoding.Default;
                }
                file.Close();
                file.Dispose();
            }
            catch
            {
                return null;
            }
            return enc;
        }
 
        /// <summary>
        /// Supprime les tags BOM d'un document
        /// </summary>
        /// <param name="ntfs">Chemin de fichier à traiter</param>
        public void DeleteTagBom(string ntfs)
        {
            FileInfo oFileInfo = new FileInfo(ntfs);
            byte[] oData = new byte[oFileInfo.Length];
            byte[] raw = new byte[oFileInfo.Length - 3];

            using (FileStream oFileStream = new FileStream(ntfs, FileMode.Open))
            {
                oFileStream.Read(oData, 0, (int)oFileInfo.Length);
                oFileStream.Close();
            }
            using (MemoryStream fs = new MemoryStream())
            {
                fs.Write(oData, 3, oData.Length - 3);
                fs.Read(raw, 0, raw.Length);
                fs.Close();
            }
            using (Stream stream = File.Create(ntfs, raw.Length, FileOptions.None))
            {
                stream.Write(raw, 0, raw.Length);
                stream.Close();
            }
        }
        
        /// <summary>
        /// Detecte la présente de tag BOM dans un document
        /// </summary>
        /// <param name="ntfs">Chemin de fichier à traiter</param>
        /// <returns>bool</returns>
        public bool DetectTagBom(string ntfs)
        {
            byte[] oData = new byte[3];
            byte[] oBom = new byte[] { 239, 187, 191 }; // i - » - ¿

            using (FileStream oFileStream = new FileStream(ntfs, FileMode.Open))
            {
                oFileStream.Read(oData, 0, 3);
                oFileStream.Close();
            }

            for (int i = 0; i < oData.Length; i++)
            {
                if (oData[i] != oBom[i])
                    return false;
            }
            
            return true;
        }

        #endregion

                
        #region 'HTML functions'
        
        /// <summary>
        /// Evalue le tag html <![CDATA[<robot>]]>
        /// et retourne un tableau qui représente la politique de "crawl"
        /// de la page web
        /// </summary>
        /// <param name="bluebot">Tag pour le robot BlueCurve</param>
        /// <param name="robot">Tag pour les autres robots</param>
        /// <returns>policy</returns>
        public bool[] CheckHtmlRobotMetaTag(string bluebot, string robot)
        {
            // index, follow
            bool[] index = { true, true };
            string[] policy = null;
            try
            {
                if (!string.IsNullOrEmpty(bluebot))
                    policy = bluebot.Split(',');
                
                if (!string.IsNullOrEmpty(robot) && string.IsNullOrEmpty(bluebot))
                    policy = robot.Split(',');

                for (int i = 0; i < policy.Length; i++)
                {
                    switch (policy[i].ToLower())
                    {
                        case "index":
                        case "follow":
                        case "all":
                            break;
                        case "noindex":
                            index[0] = false;
                            break;
                        case "nofollow":
                            index[1] = false;
                            break;
                        case "none":
                            index[0] = false;
                            index[1] = false;
                            break;
                        default:
                            break;
                    }
                }

                return index;
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// Evalue le tag html <![CDATA[<robot>]]>
        /// et retourne un tableau qui représente la politique de "crawl"
        /// de la page web
        /// </summary>
        /// <param name="ntfs">Chemin d'accès du document</param>
        /// <returns>bool[]</returns>
        public virtual bool[] CheckHtmlRobotMetaTag(string ntfs)
        {
            try
            {
                string content = null;
                using (StreamReader reader = new StreamReader(File.Open(ntfs, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite), DetectTagBom(ntfs)))
                {
                    content = reader.ReadToEnd();
                    reader.Close();
                }
                return this.CheckHtmlRobotMetaTag(this.GetHtmlMeta(ref content, "bluebot"), this.GetHtmlMeta(ref content, "robots"));
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Permet d'obtenir les tages HTML comme keywords ou description
        /// </summary>
        /// <param name="html">Contenu du document</param>
        /// <param name="meta">Meta tag a obtenir</param>
        /// <returns>string</returns>
        public string GetHtmlMeta(ref string html, string meta)
        {
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                HtmlNode node = doc.DocumentNode.SelectSingleNode("//meta[@name='" + meta + "']");
                return node.Attributes["content"].Value;
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Permet d'obtenir le titre d'un documentHTML
        /// </summary>
        /// <param name="html">Contenu du document</param>
        /// <returns>string</returns>
        public string GetHtmlTitle(ref string html)
        {
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);
                HtmlNode node = doc.DocumentNode.SelectSingleNode("html/head/title/text()");

                if (string.IsNullOrEmpty(node.InnerText))
                    node = doc.DocumentNode.SelectSingleNode("html/head/DC.Title/text()");
                
                return node.InnerText;     
            }
            catch
            {
                return null;
            }
        }
        
        /// <summary>
        /// Permet d'obtenir tout les flux RSS et ATOM
        /// d'un document HTML à partir de sont contenue
        /// </summary>
        /// <param name="content">Contenu du document</param>
        /// <returns><![CDATA[List<string>]]></returns>
        public List<string> GetRssFeeds(ref string content)
        {
            List<string> feeds = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(content);

                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//link[@rel='alternate' and @type='application/rss+xml' or @type = 'application/atom+xml']"))
                {
                    try
                    {
                        if (!feeds.Contains(node.Attributes["href"].Value))
                            feeds.Add(node.Attributes["href"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return feeds;
            }
            catch
            {
                return feeds;
            }
        }
        
        /// <summary>
        /// Permet d'obtenir tout les flux RSS et ATOM
        /// d'un document HTML à partir de sont chemin d'accès
        /// </summary>
        /// <param name="location">Chemin d'accès du document à évaluer</param>
        /// <returns><![CDATA[List<string>]]></returns>
        public virtual List<string> GetRssFeeds(string location)
        {
            List<string> feeds = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.Load(location);

                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//link[@rel='alternate' and @type='application/rss+xml' or @type = 'application/atom+xml']"))
                {
                    try
                    {
                        if (!feeds.Contains(node.Attributes["href"].Value))
                            feeds.Add(node.Attributes["href"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return feeds;
            }
            catch
            {
                return feeds;
            }
        }
        
        /// <summary>
        /// Permets d'obtenir tous les
        /// liens d'images dans un document
        /// </summary>
        /// <param name="location">Chemin d'accès du document</param>
        /// <returns><![CDATA[List<string>]]></returns>
        public List<string> GetImages(string location)
        {
            List<string> urls = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.Load(location);
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//img"))
                {
                    try
                    {
                        if (!urls.Contains(node.Attributes["src"].Value))
                        {
                            if (Convert.ToInt32(node.Attributes["height"].Value ?? "0") >= 60 && Convert.ToInt32(node.Attributes["width"].Value ?? "0") >= 60)
                            {
                                urls.Add(node.Attributes["src"].Value);
                            }
                        }
                    }
                    catch
                    {
                        continue;
                    }
                }
                return urls;
            }
            catch
            {
                return urls;
            }
        }
        
        /// <summary>
        /// Extrait les liens hypertextes d'un document HTML
        /// à partie de sont chemin d'accès
        /// </summary>
        /// <param name="location">Chemin d'accès du document à traiter</param>
        /// <returns><![CDATA[List<string>]]></returns>
        public List<string> GetHyperTextLinks(string location)
        {
            List<string> urls = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.Load(location);
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//a"))
                {
                    try
                    {
                        if (!urls.Contains(node.Attributes["href"].Value))
                            urls.Add(node.Attributes["href"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return urls;
            }
            catch
            {
                return urls;
            }
        }
        
        /// <summary>
        /// Extrait les liens hypertextes d'un document HTML
        /// à partir de sont contenu
        /// </summary>
        /// <param name="content">Contenu du document</param>
        /// <returns><![CDATA[ List<string>]]></returns>
        public virtual List<string> GetHyperTextLinks(ref string content)
        {
            List<string> urls = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(content);
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//a"))
                {
                    try
                    {
                        if (!urls.Contains(node.Attributes["href"].Value))
                            urls.Add(node.Attributes["href"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return urls;
            }
            catch
            {
                return urls;
            }
        }
        
        /// <summary>
        /// Extrait les titres (balise "title")
        /// des liens hypertextes d'un document
        /// </summary>
        /// <param name="content">Contenu du document</param>
        /// <returns><![CDATA[List<string>]]></returns>
        public List<string> GetHyperTextLinksTitle(ref string content)
        {
            List<string> urls = new List<string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(content);
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//a"))
                {
                    try
                    {
                        if (!urls.Contains(node.Attributes["title"].Value))
                            urls.Add(node.Attributes["title"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return urls;
            }
            catch
            {
                return urls;
            }
        }
        
        /// <summary>
        /// Extrait les titres (balise "title")
        /// et les liens hypertextes d'un document
        /// </summary>
        /// <param name="content">Contenu du document</param>
        /// <returns><![CDATA[Dictionary<string, string>]]></returns>
        public Dictionary<string, string> GetHyperTextLinksAndLinksTitle(ref string content)
        {
            Dictionary<string, string> urls = new Dictionary<string, string>();
            try
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(content);
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//a"))
                {
                    try
                    {
                        if (!urls.ContainsKey(node.Attributes["href"].Value))
                            urls.Add(node.Attributes["href"].Value, node.Attributes["title"].Value);
                    }
                    catch
                    {
                        continue;
                    }
                }
                return urls;
            }
            catch
            {
                return urls;
            }
        }
        
        /// <summary>
        /// Detecte l'encodage d'un document HTML
        /// à partir du tag "charset"
        /// </summary>
        public Encoding DetectHtmlEncoding(string html)
        {
            try
            {
                HtmlDocument doc = new HtmlDocument();
                return doc.DetectEncodingHtml(html);
            }
            catch
            {
                return null;
            }
        }

        #endregion

        
        #region 'XML functions'
        
        /// <summary>
        /// Convertie un document XML en texte
        /// </summary>
        /// <returns>string</returns>
        public string Cleanxml(string xml)
        {
            try
            {
                Regex clean = new Regex(@"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\+[0-9]{2}:[0-9]{2}", this.options);
                xml = clean.Replace(xml, " ");
                clean = new Regex(@"<(updated|id|published|pubDate|email|generator).*>", this.options);
                return xml = clean.Replace(xml, " ");
            }
            catch
            {
                return null;
            }
        }

        /// <summary>
        /// Convertie un document XML en texte
        /// </summary>
        /// <param name="ntfs">Chemin d'accès du fichier à traiter</param>
        /// <returns>string</returns>
        public string Convertxml2text(string ntfs)
        {
            string fs = Cleanxml(ReadFile(ntfs));
            string result = null;
            try
            {
                XmlTextReader reader = new XmlTextReader(@ntfs);
                while (reader.Read())
                {
                    switch (reader.NodeType)
                    {
                        case XmlNodeType.CDATA:
                            string s1 = reader.Value;
                            result = string.Concat(result, s1);
                            break;
                        case XmlNodeType.Text:
                            string s = reader.Value;
                            result = string.Concat(result, s);
                            break;
                        default:
                            break;
                    }
                }

                return Stripspaces(result);
            }
            catch
            {
               return null;
            }
        }

        #endregion
    }
}
